{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### News"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Placeholder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Placeholder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lRVPfEMf6CR8"
      },
      "source": [
        "### Unsloth\n",
        "\n",
        "`FastModel` supports loading nearly any model now! This includes Vision and Text models!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QOy_0RwSkN-r",
        "outputId": "08410834-5120-4735-dbf0-555e508e018b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
            "🦥 Unsloth Zoo will now patch everything to make training faster!\n",
            "env: UNSLOTH_DISABLE_FAST_GENERATION=1\n",
            "==((====))==  Unsloth 2025.8.7: Fast Modernbert patching. Transformers: 4.55.1.\n",
            "   \\\\   /|    Tesla T4. Num GPUs = 1. Max memory: 14.741 GB. Platform: Linux.\n",
            "O^O/ \\_/ \\    Torch: 2.6.0+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.2.0\n",
            "\\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.29.post3. FA2 = False]\n",
            " \"-____-\"     Free license: http://github.com/unslothai/unsloth\n",
            "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n",
            "Unsloth: Float16 full finetuning uses more memory since we upcast weights to float32.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of ModernBertForSequenceClassification were not initialized from the model checkpoint at answerdotai/ModernBERT-large and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "from unsloth import FastModel\n",
        "from transformers import AutoModelForSequenceClassification\n",
        "import torch\n",
        "\n",
        "# Disable fast generation for bert!\n",
        "%env UNSLOTH_DISABLE_FAST_GENERATION = 1\n",
        "\n",
        "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n",
        "dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n",
        "load_in_4bit = False # Use 4bit quantization to reduce memory usage. Can be False.\n",
        "\n",
        "# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n",
        "fourbit_models = [\n",
        "    \"unsloth/Meta-Llama-3.1-8B-bnb-4bit\",      # Llama-3.1 15 trillion tokens model 2x faster!\n",
        "    \"unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit\",\n",
        "    \"unsloth/Meta-Llama-3.1-70B-bnb-4bit\",\n",
        "    \"unsloth/Meta-Llama-3.1-405B-bnb-4bit\",    # We also uploaded 4bit for 405b!\n",
        "    \"unsloth/Mistral-Nemo-Base-2407-bnb-4bit\", # New Mistral 12b 2x faster!\n",
        "    \"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\",\n",
        "    \"unsloth/mistral-7b-v0.3-bnb-4bit\",        # Mistral v3 2x faster!\n",
        "    \"unsloth/mistral-7b-instruct-v0.3-bnb-4bit\",\n",
        "    \"unsloth/Phi-3.5-mini-instruct\",           # Phi-3.5 2x faster!\n",
        "    \"unsloth/Phi-3-medium-4k-instruct\",\n",
        "    \"unsloth/gemma-2-9b-bnb-4bit\",\n",
        "    \"unsloth/gemma-2-27b-bnb-4bit\",            # Gemma 2x faster!\n",
        "] # More models at https://huggingface.co/unsloth\n",
        "\n",
        "id2label = {0: \"sadness\", 1: \"joy\", 2: \"love\", 3: \"anger\",4: \"fear\",5: \"surprise\"}\n",
        "\n",
        "label2id = {\"sadness\": 0, \"joy\": 1, \"love\": 2, \"anger\": 3, \"fear\": 4, \"surprise\": 5}\n",
        "\n",
        "model, tokenizer = FastModel.from_pretrained(\n",
        "    model_name = \"answerdotai/ModernBERT-large\",\n",
        "    auto_model = AutoModelForSequenceClassification,\n",
        "    max_seq_length = max_seq_length,\n",
        "    dtype = dtype,\n",
        "    num_labels  = 6,\n",
        "    full_finetuning = True,\n",
        "    id2label=id2label,\n",
        "    label2id=label2id,\n",
        "    load_in_4bit = load_in_4bit,\n",
        "    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SB70uYYu6Jcm"
      },
      "source": [
        "We now add LoRA adapters so we only need to update a small amount of parameters!\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3gfwKIll8YO",
        "outputId": "6b956603-12ee-4c9a-ed28-8503e366f5c4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Unsloth: Full finetuning is enabled, so .get_peft_model has no effect\n"
          ]
        }
      ],
      "source": [
        "model = FastModel.get_peft_model(\n",
        "    model,\n",
        "    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n",
        "      target_modules = [\n",
        "        \"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
        "        \"gate_proj\", \"up_proj\", \"down_proj\",],\n",
        "    lora_alpha = 16,\n",
        "    lora_dropout = 0, # Supports any, but = 0 is optimized\n",
        "    bias = \"none\",    # Supports any, but = \"none\" is optimized\n",
        "    # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n",
        "    use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n",
        "    random_state = 3407,\n",
        "    use_rslora = False,  # We support rank stabilized LoRA\n",
        "    loftq_config = None, # And LoftQ\n",
        "    task_type=\"SEQ_CLS\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fqGqhTHDkN-s"
      },
      "source": [
        "<a name=\"Data\"></a>\n",
        "### Data Prep  \n",
        "We now use the [Emotion dataset](https://huggingface.co/datasets/dair-ai/emotion) from `dair-ai`, which contains text labeled by emotion. In this example, we load the **unsplit** version and use only the first 30,000 samples.  \n",
        "\n",
        "We then split the dataset into training (80%) and validation (20%), and apply tokenization to prepare the text for training.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "3913099c91b44dd68664721207725bda",
            "84caddabc29142a791b6aa7b6d07880c",
            "c936d4395a674fd7a3622ebd19e6df74",
            "87ea3acc08014b96890d9179224bb13a",
            "8d84aed419014e468a7ef9b43f58bdd0",
            "3f6c8d3399ac4fbb8eb2d3a613e8b233",
            "6b9f9baa64a14e8ab5b36e5d9d5c7588",
            "e96fcd287a2b46208aa5de21a6a8d8af",
            "20c701cfac604fd68d8276da3c7d57f3",
            "f4bf05423c4446ad88d1404d09b02e5c",
            "5c6259da9b6f43ac8b353f0e1ccd65af",
            "1b74a2a274454ec7bd37fb1ad54d0091",
            "187b99fbc9184d7c9582852f3f4df2dc",
            "64871bff18824c2bb6b518d9efab6baa",
            "d473cbb57b70434dbf9ff91a6aeb2b58",
            "03e3ad560874455489c7ab69f89642a9",
            "8dbb7880282a46bca975831af4938a7a",
            "2825026f34ba4a2587a666f7506e2d9d",
            "dfe9107c3aa04957842084431d371bec",
            "157803c9a9ad44b1a1eee666996b5098",
            "0775ba4f85ea4be18fa887a44493c940",
            "6d430404163346bf81d9994eceade43f"
          ]
        },
        "id": "lvJ4AUp-kN-t",
        "outputId": "5e8242ec-a440-46d7-b0bb-516d83baf31c"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3913099c91b44dd68664721207725bda",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map:   0%|          | 0/24000 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1b74a2a274454ec7bd37fb1ad54d0091",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map:   0%|          | 0/6000 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "# Load the IMDB dataset\n",
        "dataset = load_dataset(\"dair-ai/emotion\",\"unsplit\",split='train[:30000]')\n",
        "\n",
        "# Split into training and validation sets\n",
        "dataset = dataset.train_test_split(test_size=0.2)\n",
        "\n",
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples[\"text\"])\n",
        "\n",
        "# Apply the tokenizer to the dataset\n",
        "train_dataset = dataset['train'].map(tokenize_function, batched=True)\n",
        "val_dataset = dataset[\"test\"].map(tokenize_function, batched=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1MogHCKe0JNw"
      },
      "source": [
        "\n",
        "We compute **class weights** using scikit-learn’s ```compute_class_weight```.  \n",
        "This is useful when training on datasets where certain classes are underrepresented, ensuring the model does not become biased towards majority labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "3NAbnE4ZlpOz"
      },
      "outputs": [],
      "source": [
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "import numpy as np\n",
        "\n",
        "labels = train_dataset[\"label\"]\n",
        "class_weights = compute_class_weight(\"balanced\", classes=np.unique(labels), y=labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "7h0AmObGUfFR"
      },
      "outputs": [],
      "source": [
        "# We rename the dataset column from **`label`** to **`labels`**, since this is the expected field name for Hugging Face `Trainer`.\n",
        "train_dataset = train_dataset.rename_column(\"label\", \"labels\")\n",
        "val_dataset = val_dataset.rename_column(\"label\", \"labels\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cBMqliVdTRfZ",
        "outputId": "bb6b0496-c458-41e8-bd9d-dca90749a9f5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([0.56116723, 0.49943813, 1.998002  , 1.23839009, 1.45932142,\n",
              "       4.49438202])"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "class_weights"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4xPELDq0vum"
      },
      "source": [
        "\n",
        "We define a `compute_metrics` function to evaluate the model during training.  \n",
        "Here we use **accuracy** from scikit-learn, which compares predicted labels with the ground truth.  \n",
        "\n",
        "**[NOTE]** Accuracy is a good baseline, but for imbalanced datasets you may also want to track metrics like **F1-score**, **precision**, or **recall**.  \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "79PoG6LhX_4H"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    preds = logits.argmax(axis=-1)\n",
        "    return {\"accuracy\": accuracy_score(labels, preds)}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "271uxbcf1Jii"
      },
      "source": [
        "<a name=\"Train\"></a>\n",
        "### Train the model\n",
        "Now let's use Huggingface  `Trainer`! More docs here: [Transformers docs](https://huggingface.co/docs/transformers/main_classes/trainer). We train for one full epoch (num_train_epochs=1) to get a meaningful result."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "-Zn9nvs9kN-u"
      },
      "outputs": [],
      "source": [
        "from transformers import TrainingArguments,Trainer\n",
        "from unsloth import is_bfloat16_supported\n",
        "\n",
        "trainer = Trainer(\n",
        "    model = model,\n",
        "    processing_class = tokenizer,\n",
        "    eval_dataset = val_dataset,\n",
        "    train_dataset = train_dataset,\n",
        "    args = TrainingArguments(\n",
        "        per_device_train_batch_size = 32,\n",
        "        gradient_accumulation_steps = 1,\n",
        "        warmup_steps = 5,\n",
        "        num_train_epochs = 1, # bert-style models usually need more than 1 epoch\n",
        "        # max_steps = 60,\n",
        "        learning_rate = 5e-5,\n",
        "        fp16 = not is_bfloat16_supported(),\n",
        "        bf16 = is_bfloat16_supported(),\n",
        "        logging_steps = 1,\n",
        "        optim = \"adamw_8bit\",\n",
        "        weight_decay = 0.001,\n",
        "        eval_strategy=\"steps\",\n",
        "        eval_steps=0.10,  # Evaluate every 10% of total training steps\n",
        "        lr_scheduler_type = \"linear\",\n",
        "        seed = 3407,\n",
        "        output_dir = \"outputs\",\n",
        "        report_to = \"none\", # Use TrackIO/WandB etc\n",
        "    ),\n",
        "    compute_metrics = compute_metrics,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hGQB87Ph4Gvd"
      },
      "source": [
        "Let's train the model! To resume a training run, set `trainer.train(resume_from_checkpoint = True)`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 498
        },
        "id": "-tNmAvAHkN-u",
        "outputId": "bd4fffbb-fdb1-40a6-dcf0-6391a74dcd8e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = 1\n",
            "   \\\\   /|    Num examples = 24,000 | Num Epochs = 1 | Total steps = 750\n",
            "O^O/ \\_/ \\    Batch size per device = 32 | Gradient accumulation steps = 1\n",
            "\\        /    Data Parallel GPUs = 1 | Total batch size (32 x 1 x 1) = 32\n",
            " \"-____-\"     Trainable parameters = 395,837,446 of 395,837,446 (100.00% trained)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [750/750 13:34, Epoch 1/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>75</td>\n",
              "      <td>1.559700</td>\n",
              "      <td>1.585444</td>\n",
              "      <td>0.347000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>150</td>\n",
              "      <td>1.435100</td>\n",
              "      <td>1.583412</td>\n",
              "      <td>0.348833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>225</td>\n",
              "      <td>1.741100</td>\n",
              "      <td>1.569451</td>\n",
              "      <td>0.344167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>1.167800</td>\n",
              "      <td>1.416889</td>\n",
              "      <td>0.494333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>375</td>\n",
              "      <td>1.342500</td>\n",
              "      <td>1.258238</td>\n",
              "      <td>0.541167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>450</td>\n",
              "      <td>1.120800</td>\n",
              "      <td>1.076675</td>\n",
              "      <td>0.593167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>525</td>\n",
              "      <td>0.860900</td>\n",
              "      <td>0.940197</td>\n",
              "      <td>0.651667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>0.712700</td>\n",
              "      <td>0.820846</td>\n",
              "      <td>0.715500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>675</td>\n",
              "      <td>0.757300</td>\n",
              "      <td>0.729059</td>\n",
              "      <td>0.748833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>750</td>\n",
              "      <td>0.915100</td>\n",
              "      <td>0.688935</td>\n",
              "      <td>0.758167</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Unsloth: Will smartly offload gradients to save VRAM!\n"
          ]
        }
      ],
      "source": [
        "trainer_stats = trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0pR_Rqb31mj5"
      },
      "source": [
        "<a name=\"Inference\"></a>\n",
        "### Inference\n",
        "Let's run the model !"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rmNqG_48kN-x",
        "outputId": "a3ed6e90-1a69-4d68-89ad-8148a1b35383"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Device set to use cuda:0\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[{'label': 'joy', 'score': 0.7757943272590637}]"
            ]
          },
          "execution_count": 45,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "sentence1 = \"We just finished training ModernBERT with Unsloth and its amazing!\"\n",
        "\n",
        "classifier = pipeline(\"sentiment-analysis\", model=model,tokenizer=tokenizer)\n",
        "\n",
        "classifier(sentence1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SsjjiJ9K1vJx"
      },
      "source": [
        "<a name=\"Save\"></a>\n",
        "### Saving finetuned models\n",
        "To save the final model, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KqgqEaLe1zM6",
        "outputId": "e7d44a08-cade-4d07-cde1-48dab4c6fbe7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('model/tokenizer_config.json',\n",
              " 'model/special_tokens_map.json',\n",
              " 'model/tokenizer.json')"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.save_pretrained(\"model\")  # Local saving\n",
        "tokenizer.save_pretrained(\"model\")\n",
        "# model.push_to_hub(\"your_name/model\", token = \"...\") # Online saving\n",
        "# tokenizer.push_to_hub(\"your_name/model\", token = \"...\") # Online saving"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PkIWIXm-2Q0B"
      },
      "source": [
        "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n",
        "\n",
        "Some other links:\n",
        "1. Train your own reasoning model - Llama GRPO notebook [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb)\n",
        "2. Saving finetunes to Ollama. [Free notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)\n",
        "3. Llama 3.2 Vision finetuning - Radiography use case. [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb)\n",
        "6. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://docs.unsloth.ai/get-started/unsloth-notebooks)!\n",
        "\n",
        "<div class=\"align-center\">\n",
        "  <a href=\"https://unsloth.ai\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\" width=\"115\"></a>\n",
        "  <a href=\"https://discord.gg/unsloth\"><img src=\"https://github.com/unslothai/unsloth/raw/main/images/Discord.png\" width=\"145\"></a>\n",
        "  <a href=\"https://docs.unsloth.ai/\"><img src=\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\" width=\"125\"></a>\n",
        "\n",
        "  Join Discord if you need help + ⭐️ <i>Star us on <a href=\"https://github.com/unslothai/unsloth\">Github</a> </i> ⭐️\n",
        "</div>\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.3"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "1be15a159d9874788f7b7854451912393d9e82d0d2bc47d83a870bda7fd9bc22"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "03e3ad560874455489c7ab69f89642a9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0775ba4f85ea4be18fa887a44493c940": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "157803c9a9ad44b1a1eee666996b5098": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "187b99fbc9184d7c9582852f3f4df2dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8dbb7880282a46bca975831af4938a7a",
            "placeholder": "​",
            "style": "IPY_MODEL_2825026f34ba4a2587a666f7506e2d9d",
            "value": "Map: 100%"
          }
        },
        "1b74a2a274454ec7bd37fb1ad54d0091": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_187b99fbc9184d7c9582852f3f4df2dc",
              "IPY_MODEL_64871bff18824c2bb6b518d9efab6baa",
              "IPY_MODEL_d473cbb57b70434dbf9ff91a6aeb2b58"
            ],
            "layout": "IPY_MODEL_03e3ad560874455489c7ab69f89642a9"
          }
        },
        "20c701cfac604fd68d8276da3c7d57f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2825026f34ba4a2587a666f7506e2d9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3913099c91b44dd68664721207725bda": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_84caddabc29142a791b6aa7b6d07880c",
              "IPY_MODEL_c936d4395a674fd7a3622ebd19e6df74",
              "IPY_MODEL_87ea3acc08014b96890d9179224bb13a"
            ],
            "layout": "IPY_MODEL_8d84aed419014e468a7ef9b43f58bdd0"
          }
        },
        "3f6c8d3399ac4fbb8eb2d3a613e8b233": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c6259da9b6f43ac8b353f0e1ccd65af": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64871bff18824c2bb6b518d9efab6baa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfe9107c3aa04957842084431d371bec",
            "max": 6000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_157803c9a9ad44b1a1eee666996b5098",
            "value": 6000
          }
        },
        "6b9f9baa64a14e8ab5b36e5d9d5c7588": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d430404163346bf81d9994eceade43f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84caddabc29142a791b6aa7b6d07880c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f6c8d3399ac4fbb8eb2d3a613e8b233",
            "placeholder": "​",
            "style": "IPY_MODEL_6b9f9baa64a14e8ab5b36e5d9d5c7588",
            "value": "Map: 100%"
          }
        },
        "87ea3acc08014b96890d9179224bb13a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4bf05423c4446ad88d1404d09b02e5c",
            "placeholder": "​",
            "style": "IPY_MODEL_5c6259da9b6f43ac8b353f0e1ccd65af",
            "value": " 24000/24000 [00:06&lt;00:00, 3108.39 examples/s]"
          }
        },
        "8d84aed419014e468a7ef9b43f58bdd0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dbb7880282a46bca975831af4938a7a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c936d4395a674fd7a3622ebd19e6df74": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e96fcd287a2b46208aa5de21a6a8d8af",
            "max": 24000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_20c701cfac604fd68d8276da3c7d57f3",
            "value": 24000
          }
        },
        "d473cbb57b70434dbf9ff91a6aeb2b58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0775ba4f85ea4be18fa887a44493c940",
            "placeholder": "​",
            "style": "IPY_MODEL_6d430404163346bf81d9994eceade43f",
            "value": " 6000/6000 [00:01&lt;00:00, 5290.18 examples/s]"
          }
        },
        "dfe9107c3aa04957842084431d371bec": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e96fcd287a2b46208aa5de21a6a8d8af": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4bf05423c4446ad88d1404d09b02e5c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
